{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "tf.reset_default_graph() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n",
      "(55000, 10)\n",
      "(5000, 784)\n",
      "(5000, 10)\n",
      "(10000, 784)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets(\"./\", one_hot=True)\n",
    "\n",
    "print(mnist.train.images.shape)\n",
    "print(mnist.train.labels.shape)\n",
    "\n",
    "print(mnist.validation.images.shape)\n",
    "print(mnist.validation.labels.shape)\n",
    "\n",
    "print(mnist.test.images.shape)\n",
    "print(mnist.test.labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "第一个卷积层，使用6个5X5的卷积核对输入数据进行卷积，\n",
    "padding方式选择valid，所以输出数据的宽高变为24x24,但是深度已经从原来的1变成了6。激活函数为relu。stride为2的最大池化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_NODE = 784\n",
    "OUTPUT_NODE = 10\n",
    "IMAGE_SIZE = 28\n",
    "NUM_CHANNELS = 1\n",
    "NUM_LABELS = 10\n",
    " \n",
    "#第一层卷积层的尺寸和深度\n",
    "CONV1_DEEP = 32\n",
    "CONV1_SIZE = 5\n",
    "#第二层卷积层的尺寸和深度\n",
    "CONV2_DEEP = 64\n",
    "CONV2_SIZE = 5\n",
    "#全连接层的节点个数\n",
    "FC_SIZE = 512\n",
    " \n",
    "#\n",
    "def inference(input_tensor, train, regularizer):\n",
    "    #声明第一层卷积层的变量并实现前向传播过程\n",
    "    with tf.variable_scope('layer1-conv1'):\n",
    "        conv1_weights = tf.get_variable(\n",
    "            \"weight\", [CONV1_SIZE, CONV1_SIZE, NUM_CHANNELS, CONV1_DEEP],\n",
    "            initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        conv1_biases = tf.get_variable(\"bias\", [CONV1_DEEP], initializer=tf.constant_initializer(0.0))\n",
    "        conv1 = tf.nn.conv2d(input_tensor, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "        relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))\n",
    " \n",
    "    #声明第二层池化层的变量并实现前向传播过程\n",
    "    with tf.name_scope(\"layer2-pool1\"):\n",
    "        pool1 = tf.nn.max_pool(relu1, ksize = [1,2,2,1],strides=[1,2,2,1],padding=\"SAME\")\n",
    "    \n",
    "    #声明第三层卷积层的变量并实现前向传播过程\n",
    "    with tf.variable_scope(\"layer3-conv2\"):\n",
    "        conv2_weights = tf.get_variable(\n",
    "            \"weight\", [CONV2_SIZE, CONV2_SIZE, CONV1_DEEP, CONV2_DEEP],\n",
    "            initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        conv2_biases = tf.get_variable(\"bias\", [CONV2_DEEP], initializer=tf.constant_initializer(0.0))\n",
    "        conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1, 1, 1, 1], padding='SAME')\n",
    "        relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))\n",
    " \n",
    "    #声明第四层池化层的变量并实现前向传播过程\n",
    "    with tf.name_scope(\"layer4-pool2\"):\n",
    "        pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')\n",
    "        pool_shape = pool2.get_shape().as_list()\n",
    "        nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]\n",
    "        reshaped = tf.reshape(pool2, [pool_shape[0], nodes])\n",
    " \n",
    "    #声明第五层卷积层的变量并实现前向传播过程\n",
    "    with tf.variable_scope('layer5-fc1'):\n",
    "        fc1_weights = tf.get_variable(\"weight\", [nodes, FC_SIZE],\n",
    "                                      initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(fc1_weights))\n",
    "        fc1_biases = tf.get_variable(\"bias\", [FC_SIZE], initializer=tf.constant_initializer(0.1))\n",
    " \n",
    "        fc1 = tf.nn.relu(tf.matmul(reshaped, fc1_weights) + fc1_biases)\n",
    "        if train: fc1 = tf.nn.dropout(fc1, 0.5)\n",
    " \n",
    "    #声明第六层池化层的变量并实现前向传播过程\n",
    "    with tf.variable_scope('layer6-fc2'):\n",
    "        fc2_weights = tf.get_variable(\"weight\", [FC_SIZE, NUM_LABELS],\n",
    "                                      initializer=tf.truncated_normal_initializer(stddev=0.1))\n",
    "        if regularizer != None: tf.add_to_collection('losses', regularizer(fc2_weights))\n",
    "        fc2_biases = tf.get_variable(\"bias\", [NUM_LABELS], initializer=tf.constant_initializer(0.1))\n",
    "        logit = tf.matmul(fc1, fc2_weights) + fc2_biases\n",
    " \n",
    "    return logit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义神经网络相关的参数\n",
    "BATCH_SIZE = 100\n",
    "LEARNING_RATE_BASE = 0.01\n",
    "LEARNING_RATE_DECAY = 0.99\n",
    "REGULARIZATION_RATE = 0.0001\n",
    "TRAINING_STEPS = 6000\n",
    "MOVING_AVERAGE_DECAY = 0.99\n",
    " \n",
    "#定义训练过程\n",
    "def train(mnist):\n",
    "    # 定义输出为4维矩阵的placeholder\n",
    "    x = tf.placeholder(tf.float32, [\n",
    "            BATCH_SIZE,\n",
    "            IMAGE_SIZE,\n",
    "            IMAGE_SIZE,\n",
    "            NUM_CHANNELS],\n",
    "        name='x-input')\n",
    "    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')\n",
    "    \n",
    "    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)\n",
    "    y = inference(x,False,regularizer)\n",
    "    global_step = tf.Variable(0, trainable=False)\n",
    " \n",
    "    # 定义损失函数、学习率、滑动平均操作以及训练过程。\n",
    "    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)\n",
    "    variables_averages_op = variable_averages.apply(tf.trainable_variables())\n",
    "    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))\n",
    "    cross_entropy_mean = tf.reduce_mean(cross_entropy)\n",
    "    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))\n",
    "    learning_rate = tf.train.exponential_decay(\n",
    "        LEARNING_RATE_BASE,\n",
    "        global_step,\n",
    "        mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY,\n",
    "        staircase=True)\n",
    " \n",
    "    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)\n",
    "    with tf.control_dependencies([train_step, variables_averages_op]):\n",
    "        train_op = tf.no_op(name='train')\n",
    "        \n",
    "    # 初始化TensorFlow持久化类。\n",
    "    saver = tf.train.Saver()\n",
    "    with tf.Session() as sess:\n",
    "        tf.global_variables_initializer().run()\n",
    "        for i in range(TRAINING_STEPS):\n",
    "            xs, ys = mnist.train.next_batch(BATCH_SIZE)\n",
    " \n",
    "            reshaped_xs = np.reshape(xs, (\n",
    "                BATCH_SIZE,\n",
    "                IMAGE_SIZE,\n",
    "                IMAGE_SIZE,\n",
    "                NUM_CHANNELS))\n",
    "            _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys})\n",
    " \n",
    "            if i % 1000 == 0:\n",
    "                print(\"After %d training step(s), loss on training batch is %g.\" % (step, loss_value))\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After 1 training step(s), loss on training batch is 3.58916.\n",
      "After 1001 training step(s), loss on training batch is 0.751083.\n",
      "After 2001 training step(s), loss on training batch is 0.707012.\n",
      "After 3001 training step(s), loss on training batch is 0.684426.\n",
      "After 4001 training step(s), loss on training batch is 0.638396.\n",
      "After 5001 training step(s), loss on training batch is 0.637764.\n"
     ]
    }
   ],
   "source": [
    "train(mnist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.卷积的前向传播，经历卷积-池化-卷积-池化-全连接-全连接（防止过拟合）\n",
    "2.训练过程中，不断计算损失函数，并更新学习率，使滑动平均\n",
    "3.定义训练集与验证集，在训练轮次内循环验证，并定期打印准确率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
